{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_gan/python/estimator/tpu_gan_estimator.py:42: The name tf.estimator.tpu.TPUEstimator is deprecated. Please use tf.compat.v1.estimator.tpu.TPUEstimator instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_gan/python/estimator/tpu_gan_estimator.py:42: The name tf.estimator.tpu.TPUEstimator is deprecated. Please use tf.compat.v1.estimator.tpu.TPUEstimator instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tensor2tensor.data_generators import problem\n",
    "from tensor2tensor.data_generators import text_problems\n",
    "from tensor2tensor.data_generators import translate\n",
    "from tensor2tensor.utils import registry\n",
    "from tensor2tensor import problems\n",
    "import tensorflow as tf\n",
    "import os\n",
    "import logging\n",
    "import sentencepiece as spm\n",
    "import transformer_tag\n",
    "from tensor2tensor.layers import modalities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = 'sp10m.cased.t5.model'\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(vocab)\n",
    "\n",
    "class Encoder:\n",
    "    def __init__(self, sp):\n",
    "        self.sp = sp\n",
    "        self.vocab_size = sp.GetPieceSize() + 100\n",
    "\n",
    "    def encode(self, s):\n",
    "        return self.sp.EncodeAsIds(s)\n",
    "\n",
    "    def decode(self, ids, strip_extraneous = False):\n",
    "        return self.sp.DecodeIds(list(ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = [\n",
    "    {'class': 0, 'Description': 'PAD', 'salah': '', 'betul': ''},\n",
    "    {\n",
    "        'class': 1,\n",
    "        'Description': 'kesambungan subwords',\n",
    "        'salah': '',\n",
    "        'betul': '',\n",
    "    },\n",
    "    {'class': 2, 'Description': 'tiada kesalahan', 'salah': '', 'betul': ''},\n",
    "    {\n",
    "        'class': 3,\n",
    "        'Description': 'kesalahan frasa nama, Perkara yang diterangkan mesti mendahului \"penerang\"',\n",
    "        'salah': 'Cili sos',\n",
    "        'betul': 'sos cili',\n",
    "    },\n",
    "    {\n",
    "        'class': 4,\n",
    "        'Description': 'kesalahan kata jamak',\n",
    "        'salah': 'mereka-mereka',\n",
    "        'betul': 'mereka',\n",
    "    },\n",
    "    {\n",
    "        'class': 5,\n",
    "        'Description': 'kesalahan kata penguat',\n",
    "        'salah': 'sangat tinggi sekali',\n",
    "        'betul': 'sangat tinggi',\n",
    "    },\n",
    "    {\n",
    "        'class': 6,\n",
    "        'Description': 'kata adjektif dan imbuhan \"ter\" tanpa penguat.',\n",
    "        'salah': 'Sani mendapat markah yang tertinggi sekali.',\n",
    "        'betul': 'Sani mendapat markah yang tertinggi.',\n",
    "    },\n",
    "    {\n",
    "        'class': 7,\n",
    "        'Description': 'kesalahan kata hubung',\n",
    "        'salah': 'Sally sedang membaca bila saya tiba di rumahnya.',\n",
    "        'betul': 'Sally sedang membaca apabila saya tiba di rumahnya.',\n",
    "    },\n",
    "    {\n",
    "        'class': 8,\n",
    "        'Description': 'kesalahan kata bilangan',\n",
    "        'salah': 'Beribu peniaga tidak membayar cukai pendapatan.',\n",
    "        'betul': 'Beribu-ribu peniaga tidak membayar cukai pendapatan',\n",
    "    },\n",
    "    {\n",
    "        'class': 9,\n",
    "        'Description': 'kesalahan kata sendi',\n",
    "        'salah': 'Umar telah berpindah daripada sekolah ini bulan lalu.',\n",
    "        'betul': 'Umar telah berpindah dari sekolah ini bulan lalu.',\n",
    "    },\n",
    "    {\n",
    "        'class': 10,\n",
    "        'Description': 'kesalahan penjodoh bilangan',\n",
    "        'salah': 'Setiap orang pelajar',\n",
    "        'betul': 'Setiap pelajar.',\n",
    "    },\n",
    "    {\n",
    "        'class': 11,\n",
    "        'Description': 'kesalahan kata ganti diri',\n",
    "        'salah': 'Pencuri itu telah ditangkap. Beliau dibawa ke balai polis.',\n",
    "        'betul': 'Pencuri itu telah ditangkap. Dia dibawa ke balai polis.',\n",
    "    },\n",
    "    {\n",
    "        'class': 12,\n",
    "        'Description': 'kesalahan ayat pasif',\n",
    "        'salah': 'Cerpen itu telah dikarang oleh saya.',\n",
    "        'betul': 'Cerpen itu telah saya karang.',\n",
    "    },\n",
    "    {\n",
    "        'class': 13,\n",
    "        'Description': 'kesalahan kata tanya',\n",
    "        'salah': 'Kamu berasal dari manakah ?',\n",
    "        'betul': 'Kamu berasal dari mana ?',\n",
    "    },\n",
    "    {\n",
    "        'class': 14,\n",
    "        'Description': 'kesalahan tanda baca',\n",
    "        'salah': 'Kamu berasal dari manakah .',\n",
    "        'betul': 'Kamu berasal dari mana ?',\n",
    "    },\n",
    "    {\n",
    "        'class': 15,\n",
    "        'Description': 'kesalahan kata kerja tak transitif',\n",
    "        'salah': 'Dia kata kepada saya',\n",
    "        'betul': 'Dia berkata kepada saya',\n",
    "    },\n",
    "    {\n",
    "        'class': 16,\n",
    "        'Description': 'kesalahan kata kerja tak transitif',\n",
    "        'salah': 'Dia kata kepada saya',\n",
    "        'betul': 'Dia berkata kepada saya',\n",
    "    },\n",
    "    {\n",
    "        'class': 17,\n",
    "        'Description': 'kesalahan kata kerja transitif',\n",
    "        'salah': 'Dia suka baca buku',\n",
    "        'betul': 'Dia suka membaca buku',\n",
    "    },\n",
    "    {\n",
    "        'class': 18,\n",
    "        'Description': 'penggunaan kata yang tidak tepat',\n",
    "        'salah': 'Tembuk Besar negeri Cina dibina oleh Shih Huang Ti.',\n",
    "        'betul': 'Tembok Besar negeri Cina dibina oleh Shih Huang Ti',\n",
    "    },\n",
    "    {\n",
    "        'class': 19,\n",
    "        'Description': 'kesalahan frasa kerja tak transitif',\n",
    "        'salah': 'berdasarkan pada keterangan ini',\n",
    "        'betul': 'berdasarkan keterangan ini',\n",
    "    },\n",
    "    {\n",
    "        'class': 20,\n",
    "        'Description': 'kesalahan frasa kerja transitif',\n",
    "        'salah': 'Dia membeli banyak buah',\n",
    "        'betul': 'Dia banyak membeli buah',\n",
    "    },\n",
    "    {\n",
    "        'class': 21,\n",
    "        'Description': 'kesalahan frasa kerja pasif',\n",
    "        'salah': 'Surat itu saga akan balas',\n",
    "        'betul': 'Surat itu akan saga balas',\n",
    "    },\n",
    "]\n",
    "\n",
    "\n",
    "class Tatabahasa:\n",
    "    def __init__(self, d):\n",
    "        self.d = d\n",
    "        self.kesalahan = {i['Description']: no for no, i in enumerate(self.d)}\n",
    "        self.reverse_kesalahan = {v: k for k, v in self.kesalahan.items()}\n",
    "        self.vocab_size = len(self.d)\n",
    "\n",
    "    def encode(self, s):\n",
    "        return [self.kesalahan[i] for i in s]\n",
    "\n",
    "    def decode(self, ids, strip_extraneous = False):\n",
    "        return [self.reverse_kesalahan[i] for i in ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "@registry.register_problem\n",
    "class Grammar(text_problems.Text2TextProblem):\n",
    "    \"\"\"grammatical error correction.\"\"\"\n",
    "\n",
    "    def feature_encoders(self, data_dir):\n",
    "        encoder = Encoder(sp)\n",
    "        t = Tatabahasa(d)\n",
    "        return {'inputs': encoder, 'targets': encoder, 'targets_error_tag': t}\n",
    "\n",
    "    def hparams(self, defaults, model_hparams):\n",
    "        super(Grammar, self).hparams(defaults, model_hparams)\n",
    "        if 'use_error_tags' not in model_hparams:\n",
    "            model_hparams.add_hparam('use_error_tags', True)\n",
    "        if 'middle_prediction' not in model_hparams:\n",
    "            model_hparams.add_hparam('middle_prediction', False)\n",
    "        if 'middle_prediction_layer_factor' not in model_hparams:\n",
    "            model_hparams.add_hparam('middle_prediction_layer_factor', 2)\n",
    "        if 'ffn_in_prediction_cascade' not in model_hparams:\n",
    "            model_hparams.add_hparam('ffn_in_prediction_cascade', 1)\n",
    "        if 'error_tag_embed_size' not in model_hparams:\n",
    "            model_hparams.add_hparam('error_tag_embed_size', 12)\n",
    "        if model_hparams.use_error_tags:\n",
    "            defaults.modality[\n",
    "                'targets_error_tag'\n",
    "            ] = modalities.ModalityType.SYMBOL\n",
    "            error_tag_vocab_size = self._encoders[\n",
    "                'targets_error_tag'\n",
    "            ].vocab_size\n",
    "            defaults.vocab_size['targets_error_tag'] = error_tag_vocab_size\n",
    "\n",
    "    def example_reading_spec(self):\n",
    "        data_fields, _ = super(Grammar, self).example_reading_spec()\n",
    "        data_fields['targets_error_tag'] = tf.VarLenFeature(tf.int64)\n",
    "        return data_fields, None\n",
    "\n",
    "    @property\n",
    "    def approx_vocab_size(self):\n",
    "        return 32100\n",
    "\n",
    "    @property\n",
    "    def is_generate_per_split(self):\n",
    "        return False\n",
    "\n",
    "    @property\n",
    "    def dataset_splits(self):\n",
    "        return [\n",
    "            {'split': problem.DatasetSplit.TRAIN, 'shards': 200},\n",
    "            {'split': problem.DatasetSplit.EVAL, 'shards': 1},\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.system('mkdir t2t-tatabahasa/train-small')\n",
    "DATA_DIR = os.path.expanduser('t2t-tatabahasa/data')\n",
    "TMP_DIR = os.path.expanduser('t2t-tatabahasa/tmp')\n",
    "TRAIN_DIR = os.path.expanduser('t2t-tatabahasa/train-small')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf t2t-tatabahasa/train-small"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "PROBLEM = 'grammar'\n",
    "t2t_problem = problems.problem(PROBLEM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_steps = 200000\n",
    "eval_steps = 10\n",
    "batch_size = 1024 * 6\n",
    "save_checkpoints_steps = 25000\n",
    "ALPHA = 0.001\n",
    "schedule = 'continuous_train_and_eval'\n",
    "MODEL = 'transformer_tag'\n",
    "HPARAMS = 'transformer_base'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensor2tensor.utils.trainer_lib import create_run_config, create_experiment\n",
    "from tensor2tensor.utils.trainer_lib import create_hparams\n",
    "from tensor2tensor.utils import registry\n",
    "from tensor2tensor import models\n",
    "from tensor2tensor import problems\n",
    "\n",
    "hparams = create_hparams(HPARAMS)\n",
    "hparams.batch_size = batch_size\n",
    "hparams.learning_rate = ALPHA\n",
    "\n",
    "hparams.filter_size = 2048\n",
    "hparams.hidden_size = 512\n",
    "hparams.num_heads = 8\n",
    "hparams.num_hidden_layers = 6\n",
    "hparams.vocab_divisor = 128\n",
    "hparams.label_smoothing = 0.0\n",
    "hparams.shared_embedding_and_softmax_weights = False\n",
    "hparams.dropout = 0.1\n",
    "hparams.max_length = 1024\n",
    "# hparams.symbol_modality_num_shards = 1\n",
    "hparams.multiproblem_mixing_schedule = 'pretrain'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('activation_dtype', 'float32'), ('add_relative_to_values', False), ('area_key_mode', 'none'), ('area_value_mode', 'none'), ('attention_dropout', 0.1), ('attention_dropout_broadcast_dims', ''), ('attention_key_channels', 0), ('attention_value_channels', 0), ('attention_variables_3d', False), ('batch_shuffle_size', 512), ('batch_size', 6144), ('bottom', {}), ('causal_decoder_self_attention', True), ('clip_grad_norm', 0.0), ('compress_steps', 0), ('conv_first_kernel', 3), ('daisy_chain_variables', True), ('dropout', 0.1), ('eval_drop_long_sequences', False), ('eval_run_autoregressive', False), ('factored_logits', False), ('ffn_layer', 'dense_relu_dense'), ('filter_size', 2048), ('force_full_predict', False), ('gpu_automatic_mixed_precision', False), ('grad_noise_scale', 0.0), ('gumbel_noise_weight', 0.0), ('hard_attention_k', 0), ('heads_share_relative_embedding', False), ('hidden_size', 512), ('initializer', 'uniform_unit_scaling'), ('initializer_gain', 1.0), ('kernel_height', 3), ('kernel_width', 1), ('label_smoothing', 0.0), ('layer_postprocess_sequence', 'da'), ('layer_prepostprocess_dropout', 0.1), ('layer_prepostprocess_dropout_broadcast_dims', ''), ('layer_preprocess_sequence', 'n'), ('learning_rate', 0.001), ('learning_rate_constant', 2.0), ('learning_rate_cosine_cycle_steps', 250000), ('learning_rate_decay_rate', 1.0), ('learning_rate_decay_scheme', 'noam'), ('learning_rate_decay_staircase', False), ('learning_rate_decay_steps', 5000), ('learning_rate_minimum', None), ('learning_rate_schedule', 'rsqrt_decay'), ('learning_rate_warmup_steps', 10000), ('length_bucket_step', 1.1), ('loss', {}), ('max_area_height', 1), ('max_area_width', 1), ('max_input_seq_length', 0), ('max_length', 1024), ('max_relative_position', 0), ('max_target_seq_length', 0), ('memory_height', 1), ('min_length', 0), ('min_length_bucket', 8), ('mixed_precision_optimizer_init_loss_scale', 32768), ('mixed_precision_optimizer_loss_scaler', 'exponential'), ('mlperf_mode', False), ('moe_hidden_sizes', '2048'), ('moe_k', 2), ('moe_loss_coef', 0.001), ('moe_num_experts', 16), ('moe_overhead_eval', 2.0), ('moe_overhead_train', 1.0), ('multiply_embedding_mode', 'sqrt_depth'), ('multiproblem_fixed_train_length', -1), ('multiproblem_label_weight', 0.5), ('multiproblem_max_input_length', -1), ('multiproblem_max_target_length', -1), ('multiproblem_mixing_schedule', 'pretrain'), ('multiproblem_per_task_threshold', ''), ('multiproblem_reweight_label_loss', False), ('multiproblem_schedule_max_examples', 10000000.0), ('multiproblem_schedule_threshold', 0.5), ('multiproblem_target_eval_only', False), ('multiproblem_vocab_size', -1), ('name', {}), ('nbr_decoder_problems', 1), ('no_data_parallelism', False), ('norm_epsilon', 1e-06), ('norm_type', 'layer'), ('num_area_layers', 0), ('num_decoder_layers', 0), ('num_encoder_layers', 0), ('num_heads', 8), ('num_hidden_layers', 6), ('optimizer', 'Adafactor'), ('optimizer_adafactor_beta1', 0.0), ('optimizer_adafactor_beta2', 0.999), ('optimizer_adafactor_clipping_threshold', 1.0), ('optimizer_adafactor_decay_type', 'pow'), ('optimizer_adafactor_factored', True), ('optimizer_adafactor_memory_exponent', 0.8), ('optimizer_adafactor_multiply_by_parameter_scale', True), ('optimizer_adam_beta1', 0.9), ('optimizer_adam_beta2', 0.997), ('optimizer_adam_epsilon', 1e-09), ('optimizer_momentum_momentum', 0.9), ('optimizer_momentum_nesterov', False), ('optimizer_multistep_accumulate_steps', 0), ('optimizer_zero_grads', False), ('overload_eval_metric_name', ''), ('pack_dataset', False), ('pad_batch', False), ('parameter_attention_key_channels', 0), ('parameter_attention_value_channels', 0), ('pos', 'timing'), ('position_features', ''), ('prepend_mode', 'none'), ('pretrained_model_dir', ''), ('proximity_bias', False), ('relu_dropout', 0.1), ('relu_dropout_broadcast_dims', ''), ('sampling_keep_top_k', -1), ('sampling_method', 'argmax'), ('sampling_temp', 1.0), ('scheduled_sampling_gold_mixin_prob', 0.5), ('scheduled_sampling_method', 'parallel'), ('scheduled_sampling_num_passes', 1), ('scheduled_sampling_prob', 0.0), ('scheduled_sampling_warmup_schedule', 'exp'), ('scheduled_sampling_warmup_steps', 50000), ('self_attention_type', 'dot_product'), ('shared_embedding', False), ('shared_embedding_and_softmax_weights', False), ('split_targets_chunk_length', 0), ('split_targets_max_chunks', 100), ('split_targets_strided_training', False), ('split_to_length', 0), ('summarize_grads', False), ('summarize_vars', False), ('symbol_dropout', 0.0), ('symbol_modality_num_shards', 16), ('top', {}), ('tpu_enable_host_call', False), ('unidirectional_encoder', False), ('use_custom_ops', True), ('use_fixed_batch_size', False), ('use_pad_remover', True), ('use_target_space_embedding', True), ('video_num_input_frames', 1), ('video_num_target_frames', 1), ('vocab_divisor', 128), ('warm_start_from_second', 'small-tatabahasa/model.ckpt'), ('weight_decay', 0.0), ('weight_dtype', 'float32'), ('weight_noise', 0.0), ('weights_fn', {})]\n"
     ]
    }
   ],
   "source": [
    "hparams.optimizer = 'Adafactor'\n",
    "hparams.learning_rate_warmup_steps = 10000\n",
    "hparams.learning_rate_schedule = 'rsqrt_decay'\n",
    "hparams.warm_start_from_second = 'small-tatabahasa/model.ckpt'\n",
    "\n",
    "print(hparams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/trainer_lib.py:248: RunConfig.__init__ (from tensorflow.contrib.learn.python.learn.estimators.run_config) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "When switching to tf.estimator.Estimator, use tf.estimator.RunConfig instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/trainer_lib.py:248: RunConfig.__init__ (from tensorflow.contrib.learn.python.learn.estimators.run_config) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "When switching to tf.estimator.Estimator, use tf.estimator.RunConfig instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Configuring DataParallelism to replicate the model.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Configuring DataParallelism to replicate the model.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:schedule=continuous_train_and_eval\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:schedule=continuous_train_and_eval\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:worker_gpu=0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:worker_gpu=0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:sync=False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:sync=False\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Schedule=continuous_train_and_eval. Assuming that training is running on a single machine.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Schedule=continuous_train_and_eval. Assuming that training is running on a single machine.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:datashard_devices: ['cpu:0']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:datashard_devices: ['cpu:0']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:caching_devices: None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:caching_devices: None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:ps_devices: ['']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:ps_devices: ['']\n"
     ]
    }
   ],
   "source": [
    "RUN_CONFIG = create_run_config(\n",
    "    model_dir = TRAIN_DIR,\n",
    "    model_name = MODEL,\n",
    "    save_checkpoints_steps = 200,\n",
    "    num_gpus = 0,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fec49bc4fd0>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_train_distribute': None, '_eval_distribute': None, '_experimental_max_worker_delay_secs': None, '_device_fn': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "}\n",
      ", '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_secs': None, '_log_step_count_steps': 100, '_protocol': None, '_session_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 0.95\n",
      "}\n",
      "allow_soft_placement: true\n",
      "graph_options {\n",
      "  optimizer_options {\n",
      "    global_jit_level: OFF\n",
      "  }\n",
      "}\n",
      "isolate_session_state: true\n",
      ", '_save_checkpoints_steps': 200, '_keep_checkpoint_max': 20, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': 't2t-tatabahasa/train-small', '_session_creation_timeout_secs': 7200, 'use_tpu': False, 't2t_device_info': {'num_async_replicas': 1}, 'data_parallelism': <tensor2tensor.utils.expert_utils.Parallelism object at 0x7fec49bc4f98>}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_task_type': None, '_task_id': 0, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fec49bc4fd0>, '_master': '', '_num_ps_replicas': 0, '_num_worker_replicas': 0, '_environment': 'local', '_is_chief': True, '_evaluation_master': '', '_train_distribute': None, '_eval_distribute': None, '_experimental_max_worker_delay_secs': None, '_device_fn': None, '_tf_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 1.0\n",
      "}\n",
      ", '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_secs': None, '_log_step_count_steps': 100, '_protocol': None, '_session_config': gpu_options {\n",
      "  per_process_gpu_memory_fraction: 0.95\n",
      "}\n",
      "allow_soft_placement: true\n",
      "graph_options {\n",
      "  optimizer_options {\n",
      "    global_jit_level: OFF\n",
      "  }\n",
      "}\n",
      "isolate_session_state: true\n",
      ", '_save_checkpoints_steps': 200, '_keep_checkpoint_max': 20, '_keep_checkpoint_every_n_hours': 10000, '_model_dir': 't2t-tatabahasa/train-small', '_session_creation_timeout_secs': 7200, 'use_tpu': False, 't2t_device_info': {'num_async_replicas': 1}, 'data_parallelism': <tensor2tensor.utils.expert_utils.Parallelism object at 0x7fec49bc4f98>}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Estimator's model_fn (<function T2TModel.make_estimator_model_fn.<locals>.wrapping_model_fn at 0x7fec49c15c80>) includes params argument, but params are not passed to Estimator.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Estimator's model_fn (<function T2TModel.make_estimator_model_fn.<locals>.wrapping_model_fn at 0x7fec49c15c80>) includes params argument, but params are not passed to Estimator.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using ValidationMonitor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using ValidationMonitor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/learn/python/learn/monitors.py:279: BaseMonitor.__init__ (from tensorflow.contrib.learn.python.learn.monitors) is deprecated and will be removed after 2016-12-05.\n",
      "Instructions for updating:\n",
      "Monitors are deprecated. Please use tf.train.SessionRunHook.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/learn/python/learn/monitors.py:279: BaseMonitor.__init__ (from tensorflow.contrib.learn.python.learn.monitors) is deprecated and will be removed after 2016-12-05.\n",
      "Instructions for updating:\n",
      "Monitors are deprecated. Please use tf.train.SessionRunHook.\n"
     ]
    }
   ],
   "source": [
    "tensorflow_exp_fn = create_experiment(\n",
    "    run_config = RUN_CONFIG,\n",
    "    hparams = hparams,\n",
    "    model_name = MODEL,\n",
    "    problem_name = PROBLEM,\n",
    "    data_dir = DATA_DIR,\n",
    "    train_steps = train_steps,\n",
    "    eval_steps = eval_steps,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:EvalSpec not provided. Estimator will not manage model evaluation. Assuming ValidationMonitor present in train_hooks.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:EvalSpec not provided. Estimator will not manage model evaluation. Assuming ValidationMonitor present in train_hooks.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Reading data files from t2t-tatabahasa/data/grammar-train*\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Reading data files from t2t-tatabahasa/data/grammar-train*\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:partition: 0 num_data_files: 200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:partition: 0 num_data_files: 200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/data_generators/problem.py:689: parallel_interleave (from tensorflow.python.data.experimental.ops.interleave_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.Dataset.interleave(map_func, cycle_length, block_length, num_parallel_calls=tf.data.experimental.AUTOTUNE)` instead. If sloppy execution is desired, use `tf.data.Options.experimental_determinstic`.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/data_generators/problem.py:689: parallel_interleave (from tensorflow.python.data.experimental.ops.interleave_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.Dataset.interleave(map_func, cycle_length, block_length, num_parallel_calls=tf.data.experimental.AUTOTUNE)` instead. If sloppy execution is desired, use `tf.data.Options.experimental_determinstic`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/data_reader.py:276: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use eager execution and: \n",
      "`tf.data.TFRecordDataset(path)`\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/data_reader.py:276: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use eager execution and: \n",
      "`tf.data.TFRecordDataset(path)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/data_reader.py:38: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/data_reader.py:38: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/data/experimental/ops/grouping.py:194: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/data/experimental/ops/grouping.py:194: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/data_reader.py:234: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/data_reader.py:234: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting T2TModel mode to 'train'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting T2TModel mode to 'train'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function framework at 0x7fec54871f28> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function framework at 0x7fec54871f28> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function framework at 0x7fec54871f28> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n",
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_512.bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_512.bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_512.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_512.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets_error_tag' with symbol_modality_128_512.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets_error_tag' with symbol_modality_128_512.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:95: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:95: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function attention_bias_to_padding at 0x7fec4efc41e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function attention_bias_to_padding at 0x7fec4efc41e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function attention_bias_to_padding at 0x7fec4efc41e0> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n",
      "WARNING:tensorflow:Entity <function layers at 0x7fec542bec80> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7fec41fc3550>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function layers at 0x7fec542bec80> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7fec41fc3550>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function layers at 0x7fec542bec80> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7fec41fc3550>\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Expanding body output targets_error_tag...WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Expanding feature targets_error_tag...WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Expanding feature targets_error_tag...WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <bound method Socket.send of <zmq.sugar.socket.Socket object at 0x7fed2913c3f0>> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: <cyfunction Socket.send at 0x7fed301fbc80> is not a module, class, method, function, traceback, frame, or code object\n",
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_512.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_512.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_128_512.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_128_512.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Trainable Variables Total size: 98806804\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Trainable Variables Total size: 98806804\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Non-trainable variables Total size: 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Non-trainable variables Total size: 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using optimizer Adafactor\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using optimizer Adafactor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:optimizer names now keyed by snake_case names. Please update `registry.optimizer` callsite (likely due to a `HParams.optimizer` value)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:optimizer names now keyed by snake_case names. Please update `registry.optimizer` callsite (likely due to a `HParams.optimizer` value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Checkpoint dir: small-tatabahasa/model.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Checkpoint dir: small-tatabahasa/model.ckpt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/input_emb/weights_15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/symbol_modality_32128_512/targets/target_emb/weights_15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/symbol_modality_128_512/targets_error_tag/target_emb/weights_15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/target_space_embedding/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/target_space_embedding/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_0/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_1/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_2/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_3/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_4/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_5/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/encoder/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/layer_prepostprocess_1/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/layer_prepostprocess_1/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/layer_prepostprocess_1/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/layer_prepostprocess_1/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/edit_ops_layer/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_0/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_1/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_2/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_3/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_4/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/self_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/multihead_attention/q/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/multihead_attention/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/multihead_attention/v/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_5/ffn/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/body/decoder/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/projection/bottleneck/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/projection/bottleneck/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/projection/bottleneck/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/projection/bottleneck/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/projection/logits/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/projection/logits/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/embedding/embedding/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/embedding/embedding/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/embedding/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/embedding/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/embedding/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/error_tag_prediction/embedding/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/layer_prepostprocess/layer_norm/layer_norm_scale\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/layer_prepostprocess/layer_norm/layer_norm_bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/conv1/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/conv1/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/conv1/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/conv1/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/conv2/kernel\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/conv2/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/conv2/bias\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/body/post_error_tag/layer_0/conv2/bias\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loading variable from checkpoint: transformer_tag/targets/symbol_modality_32128_512/softmax/weights_15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_6\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_11\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Cannot find variable in checkpoint, skipping: transformer_tag/targets_error_tag/symbol_modality_128_512/softmax/weights_15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Done calling model_fn.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Done calling model_fn.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Create CheckpointSaverHook.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Create CheckpointSaverHook.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Graph was finalized.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Graph was finalized.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Running local_init_op.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saving checkpoints for 0 into t2t-tatabahasa/train-small/model.ckpt.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saving checkpoints for 0 into t2t-tatabahasa/train-small/model.ckpt.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 23.944624, step = 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 23.944624, step = 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 0.260383\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 0.260383\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.37616694, step = 101 (384.052 sec)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.37616694, step = 101 (384.052 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saving checkpoints for 200 into t2t-tatabahasa/train-small/model.ckpt.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saving checkpoints for 200 into t2t-tatabahasa/train-small/model.ckpt.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 0.281865\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 0.281865\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.41301125, step = 201 (354.779 sec)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:loss = 0.41301125, step = 201 (354.779 sec)\n"
     ]
    }
   ],
   "source": [
    "tensorflow_exp_fn.train_and_evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
